יצירת Dataset בעזרת LLMs

import boto3
import json
brt = boto3.client(service_name='bedrock-runtime')

body = json.dumps({
    "prompt": "Hello who are you",
    "maxTokens": 200,
    "temperature": 0.1,
    "topP": 1,
    "stopSequences": [],
    "countPenalty": {"scale": 0},
    "presencePenalty": {"scale": 0.8},
    "frequencyPenalty": {"scale": 0.1}
})

modelId = 'ai21.j2-ultra-v1'
accept = 'application/json'
contentType = 'application/json'

response = brt.invoke_model(body=body, modelId=modelId, accept=accept, contentType=contentType)

response_body = json.loads(response.get('body').read())
response_body

{
  "id": 1234,
  "prompt": {
    "text": "Hello who are you",
    "tokens": [
      {
        "generatedToken": {
          "token": "▁Hello",
          "logprob": -6.824674606323242,
          "raw_logprob": -6.824674606323242
        },
        "topTokens": None,
        "textRange": {
          "start": 0,
          "end": 5
        }
      },
      ...
    ]
  },
  "completions": [
    {
      "data": {
        "text": "I am Open Assistant, an open source language model trained to assist you.",
        "tokens": [
          {
            "generatedToken": {
              "token": "▁I▁am",
              "logprob": 0.0,
              "raw_logprob": -0.20282019674777985
            },
            "topTokens": None,
            "textRange": {
              "start": 0,
              "end": 4
            }
          },
          ...
        ]
      },
      "finishReason": {
        "reason": "endoftext"
      }
    }
  ]
}

import requests
import json

API_KEY = "XXXXXXXXXXXXXXXXXXXXXXXXXXXXX"
url = 'https://generativelanguage.googleapis.com/v1beta/models/gemini-pro:generateContent?key=' + API_KEY

headers = {
    'Content-Type': 'application/json',
}

data = {
    "contents": [{
        "parts": [
            {"text": "Write a story about a magic backpack."}
        ]
    }],
    "safetySettings": [
        {
            "category": "HARM_CATEGORY_DANGEROUS_CONTENT",
            "threshold": "BLOCK_ONLY_HIGH"
        }
    ],
    "generationConfig": {
        "stopSequences": [
            "Title"
        ],
        "temperature": 1.0,
        "maxOutputTokens": 800,
        "topP": 0.8,
        "topK": 10
    }
}

response = requests.post(url, headers=headers, data=json.dumps(data))
print(response.json())

{
  "candidates": [
    {
      "content": {
        "parts": [
          {
            "text": "In the bustling metropolis of Willow ...
          }
        ],
        "role": "model"
      },
      "finishReason": "STOP",
      "index": 0,
      "safetyRatings": [
        {
          "category": "HARM_CATEGORY_SEXUALLY_EXPLICIT",
          "probability": "NEGLIGIBLE"
        },
        {
          "category": "HARM_CATEGORY_HATE_SPEECH",
          "probability": "NEGLIGIBLE"
        },
        {
          "category": "HARM_CATEGORY_HARASSMENT",
          "probability": "NEGLIGIBLE"
        },
        {
          "category": "HARM_CATEGORY_DANGEROUS_CONTENT",
          "probability": "NEGLIGIBLE"
        }
      ]
    }
  ],
  "promptFeedback": {
    "safetyRatings": [
      {
        "category": "HARM_CATEGORY_SEXUALLY_EXPLICIT",
        "probability": "NEGLIGIBLE"
      },
      {
        "category": "HARM_CATEGORY_HATE_SPEECH",
        "probability": "NEGLIGIBLE"
      },
      {
        "category": "HARM_CATEGORY_HARASSMENT",
        "probability": "NEGLIGIBLE"
      },
      {
        "category": "HARM_CATEGORY_DANGEROUS_CONTENT",
        "probability": "NEGLIGIBLE"
      }
    ]
  }
}

self.default_parameters = {
    'ai21.j2-ultra-v1': {
        'maxTokens': 200,
        'temperature': 0.1,
        'topP': 1,
        'stopSequences': [],
        'countPenalty': {"scale": 0},
        'presencePenalty': {"scale": 0.8},
        'frequencyPenalty': {"scale": 0.1}
    },
    'amazon.titan-text-express-v1': {
        'maxTokenCount': 2048,
        'stopSequences': ["User:"],
        'temperature': 0.5,
        'topP': 0.9
    },
    ...
}

response = self.brt_client.invoke_model(
    body=body, 
    modelId=model_id, 
    accept='application/json', 
    contentType='application/json'
)

self.response_paths = {
    'ai21.j2-ultra-v1': ['completions', 0, 'data', 'text'],
    'amazon.titan-text-express-v1': ['results', 0, 'outputText'],
    ...
}

You are tasked with creating a single sentence that encapsulates a 
specific sentiment, given topics from specified categories. 
The sentiment is {sentiment}, with the topics:
{topics_list}
The output should be concise and limited to this sentence alone, 
with no additional explanations, comments, or queries following it. 
The response must reflect a positive outlook or outcome despite the 
context of tiredness.

Here's a sentence that fits the criteria you've described:
Assistant:

{
  "data": [
    {
      "category": "Contexts or Scenarios",
      "topics": [
        "Work environment",
        "Social events",
        "Relationship dynamics"
      ]
    },
    {
      "category": "Intensity Modifiers",
      "topics": [
        "Extreme happiness",
        "Mild annoyance",
        "Minimal interest"
      ]
    },
    ...
}

self.dynamodb.create_table(
    TableName=self.table_name,
    KeySchema=[
        {'AttributeName': 'model', 'KeyType': 'HASH'},  # Partition key
        {'AttributeName': 'timestamp', 'KeyType': 'RANGE'},  # Sort key
    ],
    AttributeDefinitions=[
        {'AttributeName': 'model', 'AttributeType': 'S'},
        {'AttributeName': 'timestamp', 'AttributeType': 'S'},
    ],
    BillingMode='PAY_PER_REQUEST'
)

 def write_item(self, model, sentiment, categories, prompt, run_time, response, request_body, full_response):
    timestamp = datetime.now(pytz.timezone('Asia/Jerusalem')).isoformat()
    item = {
        'model': model,
        'timestamp': timestamp,
        'sentiment': sentiment,
        'categories': categories,
        'prompt': prompt,
        'run_time': Decimal(str(run_time)),
        'response': response,
        'request_body': json.dumps(request_body),
        'full_response': json.dumps(full_response)
    }
    try:
        # Each item can store approximately 68,267 words (400 KB)
        self.table.put_item(Item=item)
        print(f"Item saved successfully: {model}")
    except Exception as e:
        print(f"Error saving item for {model}: {e}")

'"Although the path to emotional growth may be arduous and exhausting at 
times, it\'s important to remember that every step forward, no matter how 
small, is a step away from the limitations of our past and towards a 
brighter, more resilient future."

\n\nThis response acknowledges the challenges of emotional growth, but 
also emphasizes the importance of persevering and moving forward. It 
also incorporates a cultural proverb by referencing the idea that every 
step forward is a step away from the limitations of our past. Finally, 
it offers a positive outlook on the outcome of this process, suggesting
that emotional growth can lead to a brighter and more resilient future.'

# Add UID to each row
llama_responses = pd.DataFrame(df[df['model'] == "meta.llama2-70b-chat-v1"]["response"])
llama_responses['uid'] = range(1, len(llama_responses) + 1)
llama_responses.head()

# Split by newline and explode
df_expanded = llama_responses.set_index('uid')['response'].str.split('\n').explode().reset_index()

# Remove empty strings
df_expanded = df_expanded[df_expanded['response'].str.strip() != '']
df_expanded.head()

import requests
load_dotenv()

def get_embedding(text):
    api_url = "https://api.openai.com/v1/embeddings"
    gpt_api_key = os.getenv("OPENAI_API_KEY")
    headers = {
        "Content-Type": "application/json",
        "Authorization": f"Bearer {gpt_api_key}"
    }
    payload = {
        "input": text,
        "model": "text-embedding-3-large"
    }
    response = requests.post(api_url, json=payload, headers=headers)
    if response.status_code == 200:
        embedding = response.json()['data'][0]['embedding']
        return embedding
    else:
        return None

# Embedding Each Sentence in the DataFrame
df_expanded['embedding'] = df_expanded['response'].apply(get_embedding)

df_expanded[df_expanded['cluster'] == 0].head(10)

df_expanded[df_expanded['cluster'] == 1].head(10)

df_expanded[df_expanded['cluster'] == 2].head(25)

# First, filter rows where cluster is 0
cluster_0_df = df_expanded[df_expanded['cluster'] == 0]

# Count occurrences of each uid within the filtered DataFrame
uid_counts = cluster_0_df.groupby('uid')['uid'].transform('count')

# Unique occurrence for cluster 0
unique_occurrence = cluster_0_df[uid_counts == 1]
unique_occurrence.head()

# Filter rows where uid count is more than 1
rows_with_repeated_uids = cluster_0_df[uid_counts > 1]
rows_with_repeated_uids.head()

# Define the keywords to search for, joined by | to act as an OR operator in the regex
keywords = "this response|this sentence"

# Filter rows that do NOT contain any of the keywords, case-insensitive
rows_with_repeated_uids_no_words = rows_with_repeated_uids[
    ~rows_with_repeated_uids['response'].str.contains(keywords, case=False, regex=True)
]
rows_with_repeated_uids_no_words.head()

# Count occurrences of each uid within the filtered DataFrame
uid_counts = rows_with_repeated_uids_no_words.groupby('uid')['uid'].transform('count')
unique_occurrence_no_keywords = rows_with_repeated_uids_no_words[uid_counts == 1]
unique_occurrence_no_keywords.head()

# Filter rows where uid count is more than 1
rows_with_repeated_uids = rows_with_repeated_uids_no_words[uid_counts > 1]
rows_with_repeated_uids.head()

from sklearn.metrics import pairwise_distances

# Get the coordinates of the cluster centers
cluster_centers = kmeans.cluster_centers_

# For each point, calculate the distance to its cluster center
# First, create a function to calculate the distance
def distance_to_center(row):
    center = cluster_centers[row['cluster']]
    point = np.array([row['pca_x'], row['pca_y']])
    return np.linalg.norm(point - center)

# Apply the function to each row in the dataframe
df_expanded['distance_to_center'] = df_expanded.apply(distance_to_center, axis=1)

# Group by 'uid' and find the index of the minimum 'distance_to_center' for each group
idx = rows_with_repeated_uids.groupby('uid')['distance_to_center'].idxmin()

# Use the indices to select the rows from the original DataFrame
filtered_df_t = rows_with_repeated_uids.loc[idx]

filtered_df_t

unique_occurrence
unique_occurrence_no_keywords
filtered_df_t

# Copy the DataFrames and add a source column to each
unique_occurrence_source = unique_occurrence.copy()
unique_occurrence_source['source'] = 'unique_occurrence'

unique_occurrence_no_keywords_source = unique_occurrence_no_keywords.copy()
unique_occurrence_no_keywords_source['source'] = 'unique_occurrence_no_keywords'

filtered_df_t_source = filtered_df_t.copy()
filtered_df_t_source['source'] = 'filtered_df_t'

# Concatenate all the DataFrames vertically
all_dfs = pd.concat([
    unique_occurrence_source, 
    unique_occurrence_no_keywords_source, 
    filtered_df_t_source
])

# Drop duplicates based on 'uid' and 'response', keeping the first occurrence
slim_merged_df = all_dfs.drop_duplicates(subset=['uid', 'response'], keep='first')

# Select only the 'uid', 'response', and 'source' columns
slim_merged_df = slim_merged_df[['uid', 'response', 'source']]
slim_merged_df.sample(5)

# Get the UIDs from df_expanded
uids_expanded = df_expanded['uid'].unique()

# Check if each UID in df_expanded is in slim_merged_df
uids_not_in_slim = [uid for uid in uids_expanded if uid not in slim_merged_df['uid'].unique()]

# Print the UIDs that are not in slim_merged_df
print(uids_not_in_slim)

[]

source_responses = pd.concat(
    [other_responses, llama_cleaned], 
    axis=0
).reset_index(drop=True)
source_responses.head()

from datasets import Dataset, DatasetDict
import pandas as pd

# Shuffle the dataframe
shuffled_responses = source_responses.sample(frac=1).reset_index(drop=True)

# Map labels to numbers
label_mapping = {'positive': 1, 'negative': 0}
shuffled_responses['sentiment'] = shuffled_responses['sentiment'].map(label_mapping)
shuffled_responses = shuffled_responses.rename(columns={'response': 'text', 'sentiment': 'labels'})

shuffled_responses.head()

from datasets import Dataset, DatasetDict, Features, ClassLabel, Value

# Define dataset features, including label descriptions
features = Features({
    'text': Value('string'),
    'labels': ClassLabel(names=['negative', 'positive'])
})

# Split the shuffled dataframe (90% for training, 10% for testing)
threshold = 0.9 
split_index = int(threshold * len(shuffled_responses))  
df_train = shuffled_responses[:split_index]
df_test = shuffled_responses[split_index:]

# Convert to Dataset with defined features
train_dataset = Dataset.from_pandas(df_train, features=features)
test_dataset = Dataset.from_pandas(df_test, features=features)

# Create DatasetDict
sentiments_dataset = DatasetDict({
    'train': train_dataset,
    'test': test_dataset
})

sentiments_dataset

DatasetDict({
    train: Dataset({
        features: ['text', 'labels'],
        num_rows: 900
    }),
    test: Dataset({
        features: ['text', 'labels'],
        num_rows: 99
    })
})